{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Weight Initialization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook, we experiment with weight initialization approaches to examine their impact on layer outputs."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/the-deep-learners/deep-learning-illustrated/blob/master/notebooks/weight_initialization.ipynb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from keras import Sequential\n",
    "from keras.layers import Dense, Activation\n",
    "from keras.initializers import Zeros, RandomNormal, glorot_normal, glorot_uniform"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Set number of neurons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_input = 784\n",
    "n_dense = 256"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Select a bias initializer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "b_init = Zeros()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Select a weight initializer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "w_init = RandomNormal(stddev=1.0)\n",
    "# w_init = glorot_normal()\n",
    "# w_init = glorot_uniform()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Design neural network architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(n_dense, \n",
    "                input_dim=n_input,\n",
    "                kernel_initializer=w_init,\n",
    "                bias_initializer=b_init))\n",
    "model.add(Activation('sigmoid'))\n",
    "# model.add(Activation('tanh'))\n",
    "# model.add(Activation('relu'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Generate random *x* values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.random.random((1,n_input))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Forward propagate through the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = model.predict(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.0000000e+00, 5.5238861e-01, 4.6392031e-02, 9.9999976e-01,\n",
       "        1.0000000e+00, 1.4952397e-15, 4.9232862e-09, 1.7406979e-03,\n",
       "        9.9521768e-01, 9.9945706e-01, 1.0000000e+00, 1.4572564e-05,\n",
       "        9.9998462e-01, 9.2851160e-06, 5.8953427e-03, 1.0000000e+00,\n",
       "        1.2721360e-04, 9.9994528e-01, 9.9999118e-01, 1.5221970e-07,\n",
       "        1.0000000e+00, 2.6629050e-03, 9.6867204e-01, 1.0450470e-04,\n",
       "        9.9999988e-01, 1.0000000e+00, 1.0000000e+00, 4.4611039e-05,\n",
       "        1.0000000e+00, 9.9023002e-01, 9.9999630e-01, 3.5799325e-01,\n",
       "        9.8717526e-02, 9.9943811e-01, 2.1561574e-02, 1.0000000e+00,\n",
       "        1.0000000e+00, 1.0000000e+00, 9.9432194e-01, 9.9999595e-01,\n",
       "        9.9999988e-01, 6.7618352e-01, 8.8485974e-09, 9.9743015e-01,\n",
       "        9.3088049e-01, 4.0154563e-17, 9.9763846e-01, 2.8106602e-05,\n",
       "        1.0000000e+00, 2.9807379e-03, 1.0000000e+00, 9.2889160e-01,\n",
       "        6.6522352e-04, 1.0000000e+00, 6.3692471e-03, 1.0000000e+00,\n",
       "        3.6966151e-01, 4.0858957e-01, 2.0483537e-09, 9.2950016e-01,\n",
       "        9.9942172e-01, 8.2856990e-05, 1.8322708e-02, 9.9996555e-01,\n",
       "        1.0000000e+00, 9.8420717e-02, 1.8614588e-02, 9.9999988e-01,\n",
       "        2.2018812e-08, 9.7701776e-01, 7.5913566e-18, 9.9978286e-01,\n",
       "        1.0000000e+00, 7.5348601e-04, 1.0000000e+00, 1.2363359e-12,\n",
       "        6.0235611e-06, 4.7180316e-01, 1.8529829e-02, 1.0000000e+00,\n",
       "        9.9999964e-01, 1.0000000e+00, 3.9451379e-08, 2.0269521e-03,\n",
       "        9.9996245e-01, 3.3361158e-07, 9.9827659e-01, 9.9943238e-01,\n",
       "        8.7561755e-05, 2.2633072e-02, 7.2024915e-05, 5.6847539e-14,\n",
       "        9.9705923e-01, 1.6852890e-05, 1.5257297e-04, 1.0000000e+00,\n",
       "        2.7194526e-02, 1.0000000e+00, 2.7174382e-03, 8.1971739e-06,\n",
       "        2.0326218e-06, 1.0000000e+00, 9.9998903e-01, 1.9575008e-04,\n",
       "        1.0000000e+00, 4.7186326e-12, 9.9991298e-01, 9.9933225e-01,\n",
       "        8.4125223e-10, 2.7218473e-03, 6.6422473e-10, 2.5780533e-08,\n",
       "        9.8619580e-01, 1.0000000e+00, 4.1681342e-04, 9.8714381e-01,\n",
       "        1.2834491e-01, 2.8034039e-02, 2.1145610e-09, 9.5706326e-01,\n",
       "        7.3693615e-01, 1.9472601e-16, 1.0000000e+00, 9.9999928e-01,\n",
       "        1.4348045e-02, 7.9118448e-01, 1.0000000e+00, 4.9731020e-02,\n",
       "        5.0149702e-05, 9.9999511e-01, 2.2711025e-10, 9.9990797e-01,\n",
       "        9.3409210e-07, 6.1674172e-01, 3.4679439e-02, 4.2722717e-01,\n",
       "        9.9988282e-01, 3.2072565e-09, 3.8438510e-13, 1.0000000e+00,\n",
       "        4.3462841e-03, 1.0000000e+00, 1.0000000e+00, 4.4942368e-02,\n",
       "        9.9992895e-01, 1.0000000e+00, 1.0538031e-03, 4.4817843e-05,\n",
       "        2.4430894e-03, 1.0000000e+00, 3.3600496e-03, 9.9820447e-01,\n",
       "        9.9999702e-01, 1.0000000e+00, 1.3019590e-01, 1.0492328e-08,\n",
       "        9.7731556e-09, 1.2109972e-08, 9.4733261e-02, 7.7213299e-05,\n",
       "        9.9796945e-01, 2.0845806e-07, 6.5383670e-06, 1.0000000e+00,\n",
       "        9.9985802e-01, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00,\n",
       "        7.2833377e-01, 6.7128653e-06, 1.0000000e+00, 9.9967039e-01,\n",
       "        9.9999976e-01, 9.9307090e-01, 9.9841213e-01, 1.8684148e-05,\n",
       "        9.1203479e-03, 1.0000000e+00, 1.0000000e+00, 2.6149540e-15,\n",
       "        9.9814701e-01, 1.0000000e+00, 9.9836522e-01, 9.9999952e-01,\n",
       "        1.6699494e-04, 9.9999094e-01, 4.1960811e-06, 1.0000000e+00,\n",
       "        2.2257270e-06, 9.9976629e-01, 5.0926348e-03, 9.4292140e-01,\n",
       "        4.0563256e-03, 9.9999344e-01, 9.9880111e-01, 1.0000000e+00,\n",
       "        9.9985719e-01, 1.0000000e+00, 1.0000000e+00, 3.4722641e-10,\n",
       "        7.3154521e-01, 2.2050151e-07, 9.9904209e-01, 7.1207082e-01,\n",
       "        9.9961627e-01, 1.4136040e-10, 9.9999905e-01, 1.0000000e+00,\n",
       "        1.0000000e+00, 2.9351779e-09, 1.4806803e-05, 9.9813843e-01,\n",
       "        2.7165061e-01, 2.9061463e-01, 3.0322719e-01, 4.5177204e-14,\n",
       "        2.2284472e-09, 9.9999976e-01, 2.0912129e-02, 6.4872201e-09,\n",
       "        1.9869606e-10, 9.9999988e-01, 1.3954538e-07, 9.9999976e-01,\n",
       "        1.0000000e+00, 1.0000000e+00, 2.4544438e-03, 1.0000000e+00,\n",
       "        9.9918777e-01, 1.0000000e+00, 1.8381689e-02, 9.9999869e-01,\n",
       "        1.0000000e+00, 4.1949230e-01, 1.9061002e-03, 9.8764656e-07,\n",
       "        9.9145448e-01, 1.0000000e+00, 3.7591720e-01, 9.9999976e-01,\n",
       "        2.2123617e-04, 9.9999976e-01, 2.5395424e-15, 7.6833153e-01,\n",
       "        5.2820357e-09, 1.0000000e+00, 9.9999964e-01, 2.5045188e-11,\n",
       "        9.9991953e-01, 5.6525227e-02, 9.9491727e-01, 1.0000000e+00,\n",
       "        9.0830754e-03, 1.2347847e-07, 2.2087467e-11, 3.1971696e-11]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAADt9JREFUeJzt3X+MZWddx/H3xy4F+WULOyV1tzolWZTaaGgmTZEEkUUtLen2j2KWgCy4cSMiIhBlkT9qNCRFVJAEwZVWFoOltaLd8ENslpKqcVemFEt/ULuWuh1b2UFo/dEILHz9456ScTO79+49987tPPt+JZt7znOec8/32Zn9zJnn3HM2VYUkqV3fM+sCJEnTZdBLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGrdh1gUAbNy4sebn52ddhiStK7feeutXq2puWL/HRdDPz8+zuLg46zIkaV1J8q+j9HPqRpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGve4uDNWkmZpfvcnZnbs+6+6dOrH8Ixekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNW5o0Ce5JsmRJHesaHtXki8luT3JXyY5Y8W2tyU5lOSeJD8zrcIlSaMZ5Yz+Q8DFx7TdBJxfVT8K/DPwNoAk5wHbgR/p9vnDJKdNrFpJ0kkbGvRVdQvwtWPa/qaqjnarB4DN3fI24KNV9Y2q+jJwCLhwgvVKkk7SJObofx74VLe8CXhgxbalrk2SNCO9gj7J24GjwEcea1qlWx1n311JFpMsLi8v9ylDknQCYwd9kh3Ay4BXVtVjYb4EnLOi22bgwdX2r6o9VbVQVQtzc3PjliFJGmKsoE9yMfBW4LKqenTFpn3A9iRPTHIusAX4x/5lSpLGNfTplUmuBV4EbEyyBFzJ4FM2TwRuSgJwoKp+saruTHI9cBeDKZ3XV9W3p1W8JGm4oUFfVa9YpfnqE/R/B/COPkVJkibHO2MlqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklq3NDn0T/eze/+xMyOff9Vl87s2JI0Ks/oJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0bGvRJrklyJMkdK9qekeSmJPd2r2d27Uny3iSHktye5IJpFi9JGm6UM/oPARcf07Yb2F9VW4D93TrAS4Et3Z9dwPsnU6YkaVxDg76qbgG+dkzzNmBvt7wXuHxF+4dr4ABwRpKzJ1WsJOnkjTtH/6yqegigez2ra98EPLCi31LXJkmakUlfjM0qbbVqx2RXksUki8vLyxMuQ5L0mHGD/iuPTcl0r0e69iXgnBX9NgMPrvYGVbWnqhaqamFubm7MMiRJw4wb9PuAHd3yDuDGFe2v7j59cxHwyGNTPJKk2Rj6PPok1wIvAjYmWQKuBK4Crk+yEzgMvLzr/kngEuAQ8Cjw2inULEk6CUODvqpecZxNW1fpW8Dr+xYlSZoc74yVpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1rlfQJ3lTkjuT3JHk2iRPSnJukoNJ7k1yXZLTJ1WsJOnkjR30STYBvwIsVNX5wGnAduCdwLuragvwdWDnJAqVJI2n79TNBuB7k2wAngw8BLwYuKHbvhe4vOcxJEk9jB30VfVvwO8ChxkE/CPArcDDVXW067YEbFpt/yS7kiwmWVxeXh63DEnSEH2mbs4EtgHnAt8PPAV46Spda7X9q2pPVS1U1cLc3Ny4ZUiShugzdfMS4MtVtVxV3wI+Bvw4cEY3lQOwGXiwZ42SpB76BP1h4KIkT04SYCtwF3AzcEXXZwdwY78SJUl99JmjP8jgouvngS9277UHeCvw5iSHgGcCV0+gTknSmDYM73J8VXUlcOUxzfcBF/Z5X0nS5HhnrCQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mN6xX0Sc5IckOSLyW5O8nzkzwjyU1J7u1ez5xUsZKkk9f3jP4PgL+uqh8Gfgy4G9gN7K+qLcD+bl2SNCNjB32SpwMvBK4GqKpvVtXDwDZgb9dtL3B53yIlSePrc0b/bGAZ+JMktyX5YJKnAM+qqocAutezJlCnJGlMfYJ+A3AB8P6qeh7wP5zENE2SXUkWkywuLy/3KEOSdCJ9gn4JWKqqg936DQyC/ytJzgboXo+stnNV7amqhapamJub61GGJOlExg76qvp34IEkP9Q1bQXuAvYBO7q2HcCNvSqUJPWyoef+bwA+kuR04D7gtQx+eFyfZCdwGHh5z2NIknroFfRV9QVgYZVNW/u8ryRpcrwzVpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1LjeQZ/ktCS3Jfl4t35ukoNJ7k1yXZLT+5cpSRrXJM7o3wjcvWL9ncC7q2oL8HVg5wSOIUkaU6+gT7IZuBT4YLce4MXADV2XvcDlfY4hSeqn7xn9e4BfB77TrT8TeLiqjnbrS8CmnseQJPUwdtAneRlwpKpuXdm8Stc6zv67kiwmWVxeXh63DEnSEH3O6F8AXJbkfuCjDKZs3gOckWRD12cz8OBqO1fVnqpaqKqFubm5HmVIkk5k7KCvqrdV1eaqmge2A5+pqlcCNwNXdN12ADf2rlKSNLZpfI7+rcCbkxxiMGd/9RSOIUka0YbhXYarqs8Cn+2W7wMunMT7SpL6885YSWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS48YO+iTnJLk5yd1J7kzyxq79GUluSnJv93rm5MqVJJ2sPmf0R4G3VNVzgYuA1yc5D9gN7K+qLcD+bl2SNCNjB31VPVRVn++W/wu4G9gEbAP2dt32Apf3LVKSNL6JzNEnmQeeBxwEnlVVD8HghwFw1nH22ZVkMcni8vLyJMqQJK2id9AneSrwF8CvVtV/jrpfVe2pqoWqWpibm+tbhiTpOHoFfZInMAj5j1TVx7rmryQ5u9t+NnCkX4mSpD76fOomwNXA3VX1+ys27QN2dMs7gBvHL0+S1NeGHvu+APg54ItJvtC1/QZwFXB9kp3AYeDl/UqUJPUxdtBX1d8BOc7mreO+ryRpsrwzVpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhrX578SlNSg+d2fmNmx77/q0pkdu2We0UtS4wx6SWqcUzc6KbP8tX5WnE7QeucZvSQ1zqCXpMZNLeiTXJzkniSHkuye1nEkSSc2lTn6JKcB7wN+ClgCPpdkX1XdNY3jzcqpOF+tteP3lyZlWhdjLwQOVdV9AEk+CmwDmgp6nRoMXK1305q62QQ8sGJ9qWuTJK2xaZ3RZ5W2+n8dkl3Arm71v5PcM+axNgJfHXPf9coxnxpOuTHnnY75JP3gKJ2mFfRLwDkr1jcDD67sUFV7gD19D5RksaoW+r7PeuKYTw2O+dSwFmOe1tTN54AtSc5NcjqwHdg3pWNJkk5gKmf0VXU0yS8DnwZOA66pqjuncSxJ0olN7REIVfVJ4JPTev8Vek//rEOO+dTgmE8NUx9zqmp4L0nSuuUjECSpcesm6Ic9UiHJE5Nc120/mGR+7aucrBHG/OYkdyW5Pcn+JCN91OrxbNRHZyS5IkklWfef0BhlzEl+tvta35nkz9a6xkkb4Xv7B5LcnOS27vv7klnUOSlJrklyJMkdx9meJO/t/j5uT3LBRAuoqsf9HwYXdP8FeDZwOvBPwHnH9Pkl4APd8nbgulnXvQZj/kngyd3y606FMXf9ngbcAhwAFmZd9xp8nbcAtwFndutnzbruNRjzHuB13fJ5wP2zrrvnmF8IXADccZztlwCfYnAP0kXAwUkef72c0X/3kQpV9U3gsUcqrLQN2Nst3wBsTbLajVvrxdAxV9XNVfVot3qAwf0K69koX2eA3wZ+B/jftSxuSkYZ8y8A76uqrwNU1ZE1rnHSRhlzAU/vlr+PY+7DWW+q6hbgayfosg34cA0cAM5Icvakjr9egn6URyp8t09VHQUeAZ65JtVNx8k+RmIngzOC9WzomJM8Dzinqj6+loVN0Shf5+cAz0ny90kOJLl4zaqbjlHG/JvAq5IsMfj03hvWprSZmepjY9bL/zA19JEKI/ZZT0YeT5JXAQvAT0y1ouk74ZiTfA/wbuA1a1XQGhjl67yBwfTNixj81va3Sc6vqoenXNu0jDLmVwAfqqrfS/J84E+7MX9n+uXNxFTza72c0Q99pMLKPkk2MPh170S/Kj3ejTJmkrwEeDtwWVV9Y41qm5ZhY34acD7w2ST3M5jL3LfOL8iO+r19Y1V9q6q+DNzDIPjXq1HGvBO4HqCq/gF4EoNn/7RqpH/v41ovQT/KIxX2ATu65SuAz1R3lWOdGjrmbhrjjxiE/Hqft4UhY66qR6pqY1XNV9U8g+sSl1XV4mzKnYhRvrf/isGFd5JsZDCVc9+aVjlZo4z5MLAVIMlzGQT98ppWubb2Aa/uPn1zEfBIVT00qTdfF1M3dZxHKiT5LWCxqvYBVzP49e4QgzP57bOruL8Rx/wu4KnAn3fXnQ9X1WUzK7qnEcfclBHH/Gngp5PcBXwb+LWq+o/ZVd3PiGN+C/DHSd7EYArjNev5xC3JtQym3jZ21x2uBJ4AUFUfYHAd4hLgEPAo8NqJHn8d/91JkkawXqZuJEljMuglqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWrc/wHXkKuXqnxwggAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f257c325b00>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "_ = plt.hist(np.transpose(a))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
